import json
import pandas as pd
import torch
from datasets import Dataset
# from modelscope import snapshot_download, 
from swanlab.integration.transformers import SwanLabCallback

from peft import LoraConfig, TaskType, get_peft_model
from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer, DataCollatorForSeq2Seq
import os
import swanlab


def dataset_jsonl_transfer(origin_path, new_path):
    """
    将原始数据集转换为大模型微调所需数据格式的新数据集
    """
    messages = []

    # 读取旧的JSONL文件
    with open(origin_path, "r") as file:
        for line in file:
            # 解析每一行的json数据
            data = json.loads(line)
            context = data["text"]
            catagory = data["category"]
            label = data["output"]
            
            # message = {
            #     "instruction": "你是一个文本分类领域的专家，你会接收到一段文本和几个潜在的分类选项，请输出文本内容的正确类型",
            #     "input": f"文本:{context},类型选型:{catagory}",
            #     "output": label,
            # }
            message = [{"role": "system", "content": "你是一个文本分类领域的专家，你会接收到一段文本和几个潜在的分类选项，请输出文本内容的正确类型"},
                {"role": "user", "content": f"文本:{context},类型选型:{catagory}"},
                {"role": "assistant", "content": label}
            ]
           
            messages.append(message)

    # 保存重构后的JSONL文件
    with open(new_path, "w", encoding="utf-8") as file:
        for message in messages:
            file.write(json.dumps(message, ensure_ascii=False) + "\n")


def process_func_old(example):
    """
    将数据集进行预处理
    """
    MAX_LENGTH = 384 
    input_ids, attention_mask, labels = [], [], []
    instruction = tokenizer(
        f"<|im_start|>system\n你是一个文本分类领域的专家，你会接收到一段文本和几个潜在的分类选项，请输出文本内容的正确类型<|im_end|>\n<|im_start|>user\n{example['input']}<|im_end|>\n<|im_start|>assistant\n",
        add_special_tokens=False,
    )
    response = tokenizer(f"{example['output']}", add_special_tokens=False)
    input_ids = instruction["input_ids"] + response["input_ids"] + [tokenizer.pad_token_id]
    attention_mask = (
        instruction["attention_mask"] + response["attention_mask"] + [1]
    )
    labels = [-100] * len(instruction["input_ids"]) + response["input_ids"] + [tokenizer.pad_token_id]
    if len(input_ids) > MAX_LENGTH:  # 做一个截断
        input_ids = input_ids[:MAX_LENGTH]
        attention_mask = attention_mask[:MAX_LENGTH]
        labels = labels[:MAX_LENGTH]
    return {"input_ids": input_ids, "attention_mask": attention_mask, "labels": labels}   

def process_func(example):
    MAX_LENGTH = 512
    input_ids, attention_mask, labels = [], [], []

    # 自动兼容 input-output 数据格式
    if isinstance(example, dict) and "input" in example and "output" in example:
        messages = {
            0: {
                "role": "system",
                "content": "你是一个文本分类领域的专家，你会接收到一段文本和几个潜在的分类选项，请输出文本内容的正确类型"
            },
            1: {"role": "user", "content": example["input"]},
            2: {"role": "assistant", "content": example["output"]},
        }
    else:
        messages = example  # 已是对话格式

    # 遍历每条消息，分别处理
    for _, msg in messages.items():
        role = msg["role"]
        content = msg["content"].strip()
        msg_text = f"<|im_start|>{role}\n{content}<|im_end|>\n"
        tokenized = tokenizer(msg_text, add_special_tokens=False)

        input_ids += tokenized["input_ids"]
        attention_mask += tokenized["attention_mask"]

        # 只有 assistant 回复参与 loss
        if role == "assistant":
            label = tokenized["input_ids"]
        else:
            label = [-100] * len(tokenized["input_ids"])
        labels += label

    # 添加 pad token
    input_ids.append(tokenizer.pad_token_id)
    attention_mask.append(1)
    labels.append(-100)

    # 截断处理
    input_ids = input_ids[:MAX_LENGTH]
    attention_mask = attention_mask[:MAX_LENGTH]
    labels = labels[:MAX_LENGTH]

    return {"input_ids": input_ids, "attention_mask": attention_mask, "labels": labels}



def predict(messages, model, tokenizer):
    device = "cuda"
    text = tokenizer.apply_chat_template(
        messages,
        tokenize=False,
        add_generation_prompt=True
    )
    model_inputs = tokenizer([text], return_tensors="pt").to(device)

    generated_ids = model.generate(
        model_inputs.input_ids,
        max_new_tokens=1024
    )
    generated_ids = [
        output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
    ]

    response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]

    print(response)

    return response

# 在modelscope上下载Qwen模型到本地目录下
# model_dir = snapshot_download("qwen/Qwen2-1.5B-Instruct", cache_dir="./", revision="master")
checkpoint = "../project/Qwen_answer/Qwen2.5-7B-Instruct" 
# Transformers加载模型权重
tokenizer = AutoTokenizer.from_pretrained(checkpoint, use_fast=False, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(checkpoint, device_map="auto", torch_dtype=torch.bfloat16)
model.enable_input_require_grads()  # 开启梯度检查点时，要执行该方法

# 加载、处理数据集和测试集
train_dataset_path = "./train.jsonl"
test_dataset_path = "./test.jsonl"

train_jsonl_new_path = "./new_train.jsonl"
# test_jsonl_new_path = "./new_test.jsonl"

# if not os.path.exists(train_jsonl_new_path):
#     dataset_jsonl_transfer(train_dataset_path, train_jsonl_new_path)
# if not os.path.exists(test_jsonl_new_path):
#     dataset_jsonl_transfer(test_dataset_path, test_jsonl_new_path)

# 得到训练集
train_df = pd.read_json(train_jsonl_new_path, lines=True)
train_ds = Dataset.from_pandas(train_df)
# def load_jsonl_to_dataset(file_path):
#     data = []
#     with open(file_path, 'r', encoding='utf-8') as f:
#         for line in f:
#             try:
#                 record = json.loads(line)[0]
#                 data.append(record)
#             except json.JSONDecodeError:
#                 print(f"解析 {file_path} 中的以下行时出错: {line.strip()}")
#     return Dataset.from_list(data)

# train_ds = load_jsonl_to_dataset(train_jsonl_new_path)
train_dataset = train_ds.map(process_func, remove_columns=train_ds.column_names)

# config = LoraConfig(
#     task_type=TaskType.CAUSAL_LM,
#     target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
#     inference_mode=False,  # 训练模式
#     r=8,  # Lora 秩
#     lora_alpha=32,  # Lora alaph，具体作用参见 Lora 原理
#     lora_dropout=0.1,  # Dropout 比例
# )
config = LoraConfig(
    task_type=TaskType.CAUSAL_LM,
    target_modules=["q_proj", "v_proj"],  # 简化目标模块
    inference_mode=False,
    r=4,  # 降低参数量
    lora_alpha=16,  # 缩放等效学习率
    lora_dropout=0.2,  # 防止过拟合
)


model = get_peft_model(model, config)

args = TrainingArguments(
    output_dir="./output/Qwen2.5-7B-Instruct",
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,
    logging_steps=10,
    num_train_epochs=2,
    save_steps=100,
    learning_rate=1e-4,
    save_on_each_node=True,
    gradient_checkpointing=True,
    report_to="none",
)
# args = TrainingArguments(
#     output_dir="./output/Qwen2",
#     per_device_train_batch_size=4,
#     gradient_accumulation_steps=4,
#     logging_steps=10,
#     num_train_epochs=2,
#     save_steps=100,
#     learning_rate=1e-4,
#     save_on_each_node=True,
#     gradient_checkpointing=True,
#     report_to="none",

#     # ✅ 添加如下3项，解决报错：
#     evaluation_strategy="steps",  # 或 "epoch"，必须与 save_strategy 保持一致
#     save_strategy="steps",        # 保持一致
#     load_best_model_at_end=True,  # 启用此功能才会出错
#     metric_for_best_model="loss", # 指定最佳模型的评估指标
#     greater_is_better=False,      # 对于 loss，越小越好
# )



swanlab_callback = SwanLabCallback(
    project="Qwen2.5-7B-Instruct-fintune",
    experiment_name="Qwen2.5-7B-Instruct",
    description="使用通义千问Qwen2.5-7B-Instruct模型在rag比赛数据集上微调。",
    config={
        "model": "Qwen2.5-7B-Instruct",
        "dataset": "My_dataset",
    }
)

trainer = Trainer(
    model=model,
    args=args,
    train_dataset=train_dataset,
    data_collator=DataCollatorForSeq2Seq(tokenizer=tokenizer, padding=True),
    callbacks=[swanlab_callback],
)

trainer.train()

# 用测试集的前10条，测试模型
# test_df = pd.read_json(test_jsonl_new_path, lines=True)[:10]

# test_text_list = []
# for index, row in test_df.iterrows():
#     instruction = row['instruction']
#     input_value = row['input']

#     messages = [
#         {"role": "system", "content": f"{instruction}"},
#         {"role": "user", "content": f"{input_value}"}
#     ]

#     response = predict(messages, model, tokenizer)
#     messages.append({"role": "assistant", "content": f"{response}"})
#     result_text = f"{messages[0]}\n\n{messages[1]}\n\n{messages[2]}"
#     test_text_list.append(swanlab.Text(result_text, caption=response))

# swanlab.log({"Prediction": test_text_list})
swanlab.finish()